#! /usr/bin/python
# -*- coding: utf-8 -*-

import urllib2
import urllib
import file_handle
from bs4 import BeautifulSoup
import re


def get_str_html(html):
    result = html.replace('\n', '').replace('\r', '').strip().replace('<br/>', '\n').replace('&nbsp;', ' ')
    return result


def get_html(url):
    request = urllib2.Request(url)
    request.add_header("user-agent", "Mozilla/5.0")
    response = urllib2.urlopen(request)
    response_data = response.read()
    return get_str_html(response_data)


def get_response(url):
    request = urllib2.Request(url)
    request.add_header("user-agent", "Mozilla/5.0")
    response = urllib2.urlopen(request)
    return response
